{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl\n",
    "import polars.selectors as cs\n",
    "from polars_ds.pipeline import Pipeline, Blueprint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Builtin Pipeline Functions\n",
    "\n",
    "To run this demo: use the latest version of polars_ds. Or the latest commit if the latest version doesn't work.\n",
    "\n",
    "Need v0.5.2 or above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 22)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ID</th><th>Gender</th><th>DOB</th><th>Lead_Creation_Date</th><th>City_Code</th><th>City_Category</th><th>Employer_Code</th><th>Employer_Category1</th><th>Employer_Category2</th><th>Monthly_Income</th><th>Customer_Existing_Primary_Bank_Code</th><th>Primary_Bank_Type</th><th>Contacted</th><th>Source</th><th>Source_Category</th><th>Existing_EMI</th><th>Loan_Amount</th><th>Loan_Period</th><th>Interest_Rate</th><th>EMI</th><th>Var1</th><th>Approved</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>i64</td><td>f64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>i64</td><td>i64</td><td>f64</td><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>&quot;APPC90493171225&quot;</td><td>&quot;Female&quot;</td><td>&quot;23/07/79&quot;</td><td>&quot;15/07/16&quot;</td><td>&quot;C10001&quot;</td><td>&quot;A&quot;</td><td>&quot;COM0044082&quot;</td><td>&quot;A&quot;</td><td>4</td><td>2000.0</td><td>&quot;B001&quot;</td><td>&quot;P&quot;</td><td>&quot;N&quot;</td><td>&quot;S122&quot;</td><td>&quot;G&quot;</td><td>0.0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td></tr><tr><td>&quot;APPD40611263344&quot;</td><td>&quot;Male&quot;</td><td>&quot;07/12/86&quot;</td><td>&quot;04/07/16&quot;</td><td>&quot;C10003&quot;</td><td>&quot;A&quot;</td><td>&quot;COM0000002&quot;</td><td>&quot;C&quot;</td><td>1</td><td>3500.0</td><td>&quot;B002&quot;</td><td>&quot;P&quot;</td><td>&quot;Y&quot;</td><td>&quot;S122&quot;</td><td>&quot;G&quot;</td><td>0.0</td><td>20000</td><td>2</td><td>13.25</td><td>953</td><td>10</td><td>0</td></tr><tr><td>&quot;APPE70289249423&quot;</td><td>&quot;Male&quot;</td><td>&quot;10/12/82&quot;</td><td>&quot;19/07/16&quot;</td><td>&quot;C10125&quot;</td><td>&quot;C&quot;</td><td>&quot;COM0005267&quot;</td><td>&quot;C&quot;</td><td>4</td><td>2250.0</td><td>&quot;B003&quot;</td><td>&quot;G&quot;</td><td>&quot;Y&quot;</td><td>&quot;S143&quot;</td><td>&quot;B&quot;</td><td>0.0</td><td>45000</td><td>4</td><td>null</td><td>null</td><td>0</td><td>0</td></tr><tr><td>&quot;APPF80273865537&quot;</td><td>&quot;Male&quot;</td><td>&quot;30/01/89&quot;</td><td>&quot;09/07/16&quot;</td><td>&quot;C10477&quot;</td><td>&quot;C&quot;</td><td>&quot;COM0004143&quot;</td><td>&quot;A&quot;</td><td>4</td><td>3500.0</td><td>&quot;B003&quot;</td><td>&quot;G&quot;</td><td>&quot;Y&quot;</td><td>&quot;S143&quot;</td><td>&quot;B&quot;</td><td>0.0</td><td>92000</td><td>5</td><td>null</td><td>null</td><td>7</td><td>0</td></tr><tr><td>&quot;APPG60994436641&quot;</td><td>&quot;Male&quot;</td><td>&quot;19/04/85&quot;</td><td>&quot;20/07/16&quot;</td><td>&quot;C10002&quot;</td><td>&quot;A&quot;</td><td>&quot;COM0001781&quot;</td><td>&quot;A&quot;</td><td>4</td><td>10000.0</td><td>&quot;B001&quot;</td><td>&quot;P&quot;</td><td>&quot;Y&quot;</td><td>&quot;S134&quot;</td><td>&quot;B&quot;</td><td>2500.0</td><td>50000</td><td>2</td><td>null</td><td>null</td><td>10</td><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 22)\n",
       "┌────────────────┬────────┬──────────┬────────────────┬───┬───────────────┬──────┬──────┬──────────┐\n",
       "│ ID             ┆ Gender ┆ DOB      ┆ Lead_Creation_ ┆ … ┆ Interest_Rate ┆ EMI  ┆ Var1 ┆ Approved │\n",
       "│ ---            ┆ ---    ┆ ---      ┆ Date           ┆   ┆ ---           ┆ ---  ┆ ---  ┆ ---      │\n",
       "│ str            ┆ str    ┆ str      ┆ ---            ┆   ┆ f64           ┆ i64  ┆ i64  ┆ i64      │\n",
       "│                ┆        ┆          ┆ str            ┆   ┆               ┆      ┆      ┆          │\n",
       "╞════════════════╪════════╪══════════╪════════════════╪═══╪═══════════════╪══════╪══════╪══════════╡\n",
       "│ APPC9049317122 ┆ Female ┆ 23/07/79 ┆ 15/07/16       ┆ … ┆ null          ┆ null ┆ 0    ┆ 0        │\n",
       "│ 5              ┆        ┆          ┆                ┆   ┆               ┆      ┆      ┆          │\n",
       "│ APPD4061126334 ┆ Male   ┆ 07/12/86 ┆ 04/07/16       ┆ … ┆ 13.25         ┆ 953  ┆ 10   ┆ 0        │\n",
       "│ 4              ┆        ┆          ┆                ┆   ┆               ┆      ┆      ┆          │\n",
       "│ APPE7028924942 ┆ Male   ┆ 10/12/82 ┆ 19/07/16       ┆ … ┆ null          ┆ null ┆ 0    ┆ 0        │\n",
       "│ 3              ┆        ┆          ┆                ┆   ┆               ┆      ┆      ┆          │\n",
       "│ APPF8027386553 ┆ Male   ┆ 30/01/89 ┆ 09/07/16       ┆ … ┆ null          ┆ null ┆ 7    ┆ 0        │\n",
       "│ 7              ┆        ┆          ┆                ┆   ┆               ┆      ┆      ┆          │\n",
       "│ APPG6099443664 ┆ Male   ┆ 19/04/85 ┆ 20/07/16       ┆ … ┆ null          ┆ null ┆ 10   ┆ 0        │\n",
       "│ 1              ┆        ┆          ┆                ┆   ┆               ┆      ┆      ┆          │\n",
       "└────────────────┴────────┴──────────┴────────────────┴───┴───────────────┴──────┴──────┴──────────┘"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_parquet(\"../examples/dependency.parquet\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sql = \"\"\"\n",
    "select\n",
    "*\n",
    ", 'TEST' as test_col\n",
    "from df\n",
    "where loan_amount is not null\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Blueprint name: example\n",
      "Blueprint current steps: 15\n",
      "Features Expected: ['ID', 'Gender', 'DOB', 'Lead_Creation_Date', 'City_Code', 'City_Category', 'Employer_Code', 'Employer_Category1', 'Employer_Category2', 'Monthly_Income', 'Customer_Existing_Primary_Bank_Code', 'Primary_Bank_Type', 'Contacted', 'Source', 'Source_Category', 'Existing_EMI', 'Loan_Amount', 'Loan_Period', 'Interest_Rate', 'EMI', 'Var1', 'Approved']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Create a blueprint first. \n",
    "# A blueprint is a plan for a pipeline. No hard work will be done until the blueprint is materialized, which\n",
    "# is when the tranforms are fitted (e.g. scale learns the mean and std from base data)\n",
    "# If target is specified for the blueprint, target will be excluded from all transformations that require a fit,\n",
    "# and target will be auto-filled if the transformation requires a target field and when no target field is explicitly given.\n",
    "\n",
    "bp = (\n",
    "    Blueprint(df, name = \"example\", target = \"approved\") # You can optionally put target of the ML model here\n",
    "    # Select only the columns we need\n",
    "    .lowercase() # lowercase all columns\n",
    "    .sql_transform(sql) # Run a SQL transform on the df\n",
    "    # Say you want to remove a population for your data pipeline.\n",
    "    .filter( \n",
    "        \"city_category is not null\" # or equivalently, you can do: pl.col(\"city_category\").is_not_null()\n",
    "    )\n",
    "    .select(cs.numeric() | cs.by_name([\"gender\", \"employer_category1\", \"city_category\", \"test_col\"]))\n",
    "    # explicitly put target, since this is not the target for prediction. \n",
    "    # Use a linear regression with x1 = var1, x2=existing_emi to predict missing values in loan_period\n",
    "    .linear_impute(features = [\"var1\", \"existing_emi\"], target = \"loan_period\") \n",
    "    .impute([\"existing_emi\"], method = \"median\")\n",
    "    .append_expr( # generate some features\n",
    "        pl.col(\"existing_emi\").log1p().alias(\"existing_emi_log1p\"),\n",
    "        pl.col(\"loan_amount\").log1p().alias(\"loan_amount_log1p\"),\n",
    "        pl.col(\"loan_amount\").clip(lower_bound = 0, upper_bound = 1000).alias(\"loan_amount_log1p_clipped\"),\n",
    "        pl.col(\"loan_amount\").sqrt().alias(\"loan_amount_sqrt\"),\n",
    "        pl.col(\"loan_amount\").shift(-1).alias(\"loan_amount_lag_1\") # any kind of lag transform\n",
    "    )\n",
    "    .scale( # target is numerical, but will be excluded automatically because bp is initialzied with a target\n",
    "        cs.numeric().exclude([\"var1\", \"existing_emi_log1p\"]), method = \"standard\"\n",
    "    ) # Scale the columns up to this point. The columns below won't be scaled\n",
    "    .append_expr(\n",
    "        # Add missing flags\n",
    "        pl.col(\"employer_category1\").is_null().cast(pl.UInt8).alias(\"employer_category1_is_missing\")\n",
    "    )\n",
    "    .one_hot_encode(\"gender\", drop_first=True)\n",
    "    .woe_encode(\"city_category\") # No need to specify target because we initialized bp with a target\n",
    "    .target_encode(\"employer_category1\", min_samples_leaf = 20, smoothing = 10.0) # same as above\n",
    "    .shrink_dtype(force_f32 = True) # shrink dtype to smallest possible. Force floats to be f32\n",
    ")\n",
    "\n",
    "print(bp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Naive Query Steps: \n",
       "\n",
       "Step 1:\n",
       "col(\"ID\").alias(\"id\"),\n",
       "col(\"Gender\").alias(\"gender\"),\n",
       "col(\"DOB\").alias(\"dob\"),\n",
       "col(\"Lead_Creation_Date\").alias(\"lead_creation_date\"),\n",
       "col(\"City_Code\").alias(\"city_code\"),\n",
       "col(\"City_Category\").alias(\"city_category\"),\n",
       "col(\"Employer_Code\").alias(\"employer_code\"),\n",
       "col(\"Employer_Category1\").alias(\"employer_category1\"),\n",
       "col(\"Employer_Category2\").alias(\"employer_category2\"),\n",
       "col(\"Monthly_Income\").alias(\"monthly_income\"),\n",
       "col(\"Customer_Existing_Primary_Bank_Code\").alias(\"customer_existing_primary_bank_code\"),\n",
       "col(\"Primary_Bank_Type\").alias(\"primary_bank_type\"),\n",
       "col(\"Contacted\").alias(\"contacted\"),\n",
       "col(\"Source\").alias(\"source\"),\n",
       "col(\"Source_Category\").alias(\"source_category\"),\n",
       "col(\"Existing_EMI\").alias(\"existing_emi\"),\n",
       "col(\"Loan_Amount\").alias(\"loan_amount\"),\n",
       "col(\"Loan_Period\").alias(\"loan_period\"),\n",
       "col(\"Interest_Rate\").alias(\"interest_rate\"),\n",
       "col(\"EMI\").alias(\"emi\"),\n",
       "col(\"Var1\").alias(\"var1\"),\n",
       "col(\"Approved\").alias(\"approved\")\n",
       "\n",
       "Step 2:\n",
       "Run SQL: \n",
       "select\n",
       "*\n",
       ", 'TEST' as test_col\n",
       "from df\n",
       "where loan_amount is not null\n",
       "\n",
       "\n",
       "\n",
       "Step 3:\n",
       "col(\"city_category\").is_not_null().all_horizontal()\n",
       "\n",
       "Step 4:\n",
       "selector\n",
       "\n",
       "Step 5:\n",
       ".when(col(\"loan_period\").is_null()).then([(col(\"var1\")) * (dyn float: 0.50981)].sum_horizontal([[(col(\"existing_emi\")) * (dyn float: -0.000008)]])).otherwise(col(\"loan_period\").strict_cast(Float64)).alias(\"loan_period\")\n",
       "\n",
       "Step 6:\n",
       "col(\"existing_emi\").fill_null([dyn float: 0.0])\n",
       "\n",
       "Step 7:\n",
       "col(\"existing_emi\").log1p().alias(\"existing_emi_log1p\"),\n",
       "col(\"loan_amount\").log1p().alias(\"loan_amount_log1p\"),\n",
       "col(\"loan_amount\").clip([dyn int: 0, dyn int: 1000]).alias(\"loan_amount_log1p_clipped\"),\n",
       "col(\"loan_amount\").sqrt().alias(\"loan_amount_sqrt\"),\n",
       "col(\"loan_amount\").shift([dyn int: -1]).alias(\"loan_amount_lag_1\")\n",
       "\n",
       "Step 8:\n",
       "[([(col(\"employer_category2\")) - (dyn float: 3.67927)]) / (dyn float: 0.862459)],\n",
       "[([(col(\"monthly_income\")) - (dyn float: 7463.79731)]) / (dyn float: 225051.544361)],\n",
       "[([(col(\"existing_emi\")) - (dyn float: 265.690248)]) / (dyn float: 2757.609254)],\n",
       "[([(col(\"loan_amount\")) - (dyn float: 39429.982859)]) / (dyn float: 30727.230218)],\n",
       "[([(col(\"loan_period\")) - (dyn float: 3.890629)]) / (dyn float: 1.167477)],\n",
       "[([(col(\"interest_rate\")) - (dyn float: 19.21357)]) / (dyn float: 5.847005)],\n",
       "[([(col(\"emi\")) - (dyn float: 1101.466242)]) / (dyn float: 752.6445)],\n",
       "[([(col(\"loan_amount_log1p\")) - (dyn float: 10.326665)]) / (dyn float: 0.721931)],\n",
       "[([(col(\"loan_amount_log1p_clipped\")) - (dyn float: 1000.0)]) / (dyn float: 0.0)],\n",
       "[([(col(\"loan_amount_sqrt\")) - (dyn float: 186.421961)]) / (dyn float: 68.387392)],\n",
       "[([(col(\"loan_amount_lag_1\")) - (dyn float: 39430.445444)]) / (dyn float: 30727.449733)]\n",
       "\n",
       "Step 9:\n",
       "col(\"employer_category1\").is_null().strict_cast(UInt8).alias(\"employer_category1_is_missing\")\n",
       "\n",
       "Step 10:\n",
       "[(col(\"gender\")) ==v (String(Male))].strict_cast(UInt8).alias(\"gender_Male\")\n",
       "\n",
       "Step 11:\n",
       "*.exclude([Name(\"gender\")])\n",
       "\n",
       "Step 12:\n",
       "col(\"city_category\").replace_strict([Series[value], Series[woe], null])\n",
       "\n",
       "Step 13:\n",
       "col(\"employer_category1\").replace_strict([Series[value], Series[to], null])\n",
       "\n",
       "Step 14:\n",
       "dtype_columns([UInt32, UInt8, Int32, Int8, UInt64, UInt16, Int64, Int16]).shrink_dtype()\n",
       "\n",
       "Step 15:\n",
       "dtype_columns([Float32, Float64]).strict_cast(Float32)\n"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Materialize the blueprint\n",
    "pipe:Pipeline = bp.materialize()\n",
    "# Text representation of the pipeline\n",
    "pipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 19)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>city_category</th><th>employer_category1</th><th>employer_category2</th><th>monthly_income</th><th>existing_emi</th><th>loan_amount</th><th>loan_period</th><th>interest_rate</th><th>emi</th><th>var1</th><th>approved</th><th>test_col</th><th>existing_emi_log1p</th><th>loan_amount_log1p</th><th>loan_amount_log1p_clipped</th><th>loan_amount_sqrt</th><th>loan_amount_lag_1</th><th>employer_category1_is_missing</th><th>gender_Male</th></tr><tr><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>i8</td><td>i8</td><td>str</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>u8</td><td>u8</td></tr></thead><tbody><tr><td>0.080959</td><td>0.026603</td><td>-3.106548</td><td>-0.017613</td><td>-0.096348</td><td>-0.632338</td><td>-1.619415</td><td>-1.019936</td><td>-0.197259</td><td>10</td><td>0</td><td>&quot;TEST&quot;</td><td>0.0</td><td>-0.586105</td><td>NaN</td><td>-0.658025</td><td>0.181257</td><td>0</td><td>1</td></tr><tr><td>-0.479553</td><td>0.026603</td><td>0.371879</td><td>-0.023167</td><td>-0.096348</td><td>0.181273</td><td>0.093681</td><td>null</td><td>null</td><td>0</td><td>0</td><td>&quot;TEST&quot;</td><td>0.0</td><td>0.537137</td><td>NaN</td><td>0.375948</td><td>1.710834</td><td>0</td><td>1</td></tr><tr><td>-0.479553</td><td>0.014737</td><td>0.371879</td><td>-0.017613</td><td>-0.096348</td><td>1.710861</td><td>0.950229</td><td>null</td><td>null</td><td>7</td><td>0</td><td>&quot;TEST&quot;</td><td>0.0</td><td>1.527696</td><td>NaN</td><td>1.709278</td><td>0.343978</td><td>0</td><td>1</td></tr><tr><td>0.080959</td><td>0.014737</td><td>0.371879</td><td>0.011269</td><td>0.810234</td><td>0.343995</td><td>-1.619415</td><td>null</td><td>null</td><td>10</td><td>0</td><td>&quot;TEST&quot;</td><td>7.824446</td><td>0.683076</td><td>NaN</td><td>0.543738</td><td>2.947513</td><td>0</td><td>1</td></tr><tr><td>-0.046475</td><td>0.024336</td><td>0.371879</td><td>0.000161</td><td>-0.096348</td><td>2.947549</td><td>0.950229</td><td>-0.746292</td><td>2.631433</td><td>10</td><td>0</td><td>&quot;TEST&quot;</td><td>0.0</td><td>2.00661</td><td>NaN</td><td>2.546276</td><td>-0.306906</td><td>0</td><td>1</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 19)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ city_cate ┆ employer_ ┆ employer_ ┆ monthly_i ┆ … ┆ loan_amou ┆ loan_amou ┆ employer_ ┆ gender_M │\n",
       "│ gory      ┆ category1 ┆ category2 ┆ ncome     ┆   ┆ nt_sqrt   ┆ nt_lag_1  ┆ category1 ┆ ale      │\n",
       "│ ---       ┆ ---       ┆ ---       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ _is_missi ┆ ---      │\n",
       "│ f32       ┆ f32       ┆ f32       ┆ f32       ┆   ┆ f32       ┆ f32       ┆ ng        ┆ u8       │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ ---       ┆          │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ u8        ┆          │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "│ 0.080959  ┆ 0.026603  ┆ -3.106548 ┆ -0.017613 ┆ … ┆ -0.658025 ┆ 0.181257  ┆ 0         ┆ 1        │\n",
       "│ -0.479553 ┆ 0.026603  ┆ 0.371879  ┆ -0.023167 ┆ … ┆ 0.375948  ┆ 1.710834  ┆ 0         ┆ 1        │\n",
       "│ -0.479553 ┆ 0.014737  ┆ 0.371879  ┆ -0.017613 ┆ … ┆ 1.709278  ┆ 0.343978  ┆ 0         ┆ 1        │\n",
       "│ 0.080959  ┆ 0.014737  ┆ 0.371879  ┆ 0.011269  ┆ … ┆ 0.543738  ┆ 2.947513  ┆ 0         ┆ 1        │\n",
       "│ -0.046475 ┆ 0.024336  ┆ 0.371879  ┆ 0.000161  ┆ … ┆ 2.546276  ┆ -0.306906 ┆ 0         ┆ 1        │\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# If you want separation between features (X) and target (y)\n",
    "# you can run the following:\n",
    "# df_x, df_y = pipe.transform(df, separate=True)\n",
    "\n",
    "df_transformed = pipe.transform(df)\n",
    "df_transformed.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (0, 19)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>city_category</th><th>employer_category1</th><th>employer_category2</th><th>monthly_income</th><th>existing_emi</th><th>loan_amount</th><th>loan_period</th><th>interest_rate</th><th>emi</th><th>var1</th><th>approved</th><th>test_col</th><th>existing_emi_log1p</th><th>loan_amount_log1p</th><th>loan_amount_log1p_clipped</th><th>loan_amount_sqrt</th><th>loan_amount_lag_1</th><th>employer_category1_is_missing</th><th>gender_Male</th></tr><tr><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>i8</td><td>i8</td><td>str</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>u8</td><td>u8</td></tr></thead><tbody></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (0, 19)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ city_cate ┆ employer_ ┆ employer_ ┆ monthly_i ┆ … ┆ loan_amou ┆ loan_amou ┆ employer_ ┆ gender_M │\n",
       "│ gory      ┆ category1 ┆ category2 ┆ ncome     ┆   ┆ nt_sqrt   ┆ nt_lag_1  ┆ category1 ┆ ale      │\n",
       "│ ---       ┆ ---       ┆ ---       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ _is_missi ┆ ---      │\n",
       "│ f32       ┆ f32       ┆ f32       ┆ f32       ┆   ┆ f32       ┆ f32       ┆ ng        ┆ u8       │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ ---       ┆          │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ u8        ┆          │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Empty. Because we filtered this to not null.\n",
    "df_transformed.filter(\n",
    "    pl.col(\"city_category\").is_null()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (0, 19)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>city_category</th><th>employer_category1</th><th>employer_category2</th><th>monthly_income</th><th>existing_emi</th><th>loan_amount</th><th>loan_period</th><th>interest_rate</th><th>emi</th><th>var1</th><th>approved</th><th>test_col</th><th>existing_emi_log1p</th><th>loan_amount_log1p</th><th>loan_amount_log1p_clipped</th><th>loan_amount_sqrt</th><th>loan_amount_lag_1</th><th>employer_category1_is_missing</th><th>gender_Male</th></tr><tr><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>i8</td><td>i8</td><td>str</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>u8</td><td>u8</td></tr></thead><tbody></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (0, 19)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ city_cate ┆ employer_ ┆ employer_ ┆ monthly_i ┆ … ┆ loan_amou ┆ loan_amou ┆ employer_ ┆ gender_M │\n",
       "│ gory      ┆ category1 ┆ category2 ┆ ncome     ┆   ┆ nt_sqrt   ┆ nt_lag_1  ┆ category1 ┆ ale      │\n",
       "│ ---       ┆ ---       ┆ ---       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ _is_missi ┆ ---      │\n",
       "│ f32       ┆ f32       ┆ f32       ┆ f32       ┆   ┆ f32       ┆ f32       ┆ ng        ┆ u8       │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ ---       ┆          │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ u8        ┆          │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Empty. Because we filtered this to not null in the SQL\n",
    "df_transformed.filter(\n",
    "    pl.col(\"loan_amount\").is_null()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Serialization Methods\n",
    "\n",
    "Pickle + JSON support."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "# The pipe object can be pickled\n",
    "with open(\"pipe.pickle\", \"wb\") as f:\n",
    "    pickle.dump(pipe, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Naive Query Steps: \n",
       "\n",
       "Step 1:\n",
       "col(\"ID\").alias(\"id\"),\n",
       "col(\"Gender\").alias(\"gender\"),\n",
       "col(\"DOB\").alias(\"dob\"),\n",
       "col(\"Lead_Creation_Date\").alias(\"lead_creation_date\"),\n",
       "col(\"City_Code\").alias(\"city_code\"),\n",
       "col(\"City_Category\").alias(\"city_category\"),\n",
       "col(\"Employer_Code\").alias(\"employer_code\"),\n",
       "col(\"Employer_Category1\").alias(\"employer_category1\"),\n",
       "col(\"Employer_Category2\").alias(\"employer_category2\"),\n",
       "col(\"Monthly_Income\").alias(\"monthly_income\"),\n",
       "col(\"Customer_Existing_Primary_Bank_Code\").alias(\"customer_existing_primary_bank_code\"),\n",
       "col(\"Primary_Bank_Type\").alias(\"primary_bank_type\"),\n",
       "col(\"Contacted\").alias(\"contacted\"),\n",
       "col(\"Source\").alias(\"source\"),\n",
       "col(\"Source_Category\").alias(\"source_category\"),\n",
       "col(\"Existing_EMI\").alias(\"existing_emi\"),\n",
       "col(\"Loan_Amount\").alias(\"loan_amount\"),\n",
       "col(\"Loan_Period\").alias(\"loan_period\"),\n",
       "col(\"Interest_Rate\").alias(\"interest_rate\"),\n",
       "col(\"EMI\").alias(\"emi\"),\n",
       "col(\"Var1\").alias(\"var1\"),\n",
       "col(\"Approved\").alias(\"approved\")\n",
       "\n",
       "Step 2:\n",
       "Run SQL: \n",
       "select\n",
       "*\n",
       ", 'TEST' as test_col\n",
       "from df\n",
       "where loan_amount is not null\n",
       "\n",
       "\n",
       "\n",
       "Step 3:\n",
       "col(\"city_category\").is_not_null().all_horizontal()\n",
       "\n",
       "Step 4:\n",
       "selector\n",
       "\n",
       "Step 5:\n",
       ".when(col(\"loan_period\").is_null()).then([(col(\"var1\")) * (dyn float: 0.50981)].sum_horizontal([[(col(\"existing_emi\")) * (dyn float: -0.000008)]])).otherwise(col(\"loan_period\").strict_cast(Float64)).alias(\"loan_period\")\n",
       "\n",
       "Step 6:\n",
       "col(\"existing_emi\").fill_null([dyn float: 0.0])\n",
       "\n",
       "Step 7:\n",
       "col(\"existing_emi\").log1p().alias(\"existing_emi_log1p\"),\n",
       "col(\"loan_amount\").log1p().alias(\"loan_amount_log1p\"),\n",
       "col(\"loan_amount\").clip([dyn int: 0, dyn int: 1000]).alias(\"loan_amount_log1p_clipped\"),\n",
       "col(\"loan_amount\").sqrt().alias(\"loan_amount_sqrt\"),\n",
       "col(\"loan_amount\").shift([dyn int: -1]).alias(\"loan_amount_lag_1\")\n",
       "\n",
       "Step 8:\n",
       "[([(col(\"employer_category2\")) - (dyn float: 3.67927)]) / (dyn float: 0.862459)],\n",
       "[([(col(\"monthly_income\")) - (dyn float: 7463.79731)]) / (dyn float: 225051.544361)],\n",
       "[([(col(\"existing_emi\")) - (dyn float: 265.690248)]) / (dyn float: 2757.609254)],\n",
       "[([(col(\"loan_amount\")) - (dyn float: 39429.982859)]) / (dyn float: 30727.230218)],\n",
       "[([(col(\"loan_period\")) - (dyn float: 3.890629)]) / (dyn float: 1.167477)],\n",
       "[([(col(\"interest_rate\")) - (dyn float: 19.21357)]) / (dyn float: 5.847005)],\n",
       "[([(col(\"emi\")) - (dyn float: 1101.466242)]) / (dyn float: 752.6445)],\n",
       "[([(col(\"loan_amount_log1p\")) - (dyn float: 10.326665)]) / (dyn float: 0.721931)],\n",
       "[([(col(\"loan_amount_log1p_clipped\")) - (dyn float: 1000.0)]) / (dyn float: 0.0)],\n",
       "[([(col(\"loan_amount_sqrt\")) - (dyn float: 186.421961)]) / (dyn float: 68.387392)],\n",
       "[([(col(\"loan_amount_lag_1\")) - (dyn float: 39430.445444)]) / (dyn float: 30727.449733)]\n",
       "\n",
       "Step 9:\n",
       "col(\"employer_category1\").is_null().strict_cast(UInt8).alias(\"employer_category1_is_missing\")\n",
       "\n",
       "Step 10:\n",
       "[(col(\"gender\")) ==v (String(Male))].strict_cast(UInt8).alias(\"gender_Male\")\n",
       "\n",
       "Step 11:\n",
       "*.exclude([Name(\"gender\")])\n",
       "\n",
       "Step 12:\n",
       "col(\"city_category\").replace_strict([Series[value], Series[woe], null])\n",
       "\n",
       "Step 13:\n",
       "col(\"employer_category1\").replace_strict([Series[value], Series[to], null])\n",
       "\n",
       "Step 14:\n",
       "dtype_columns([UInt32, UInt8, Int32, Int8, UInt64, UInt16, Int64, Int16]).shrink_dtype()\n",
       "\n",
       "Step 15:\n",
       "dtype_columns([Float32, Float64]).strict_cast(Float32)\n"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(\"pipe.pickle\", \"rb\") as f:\n",
    "    pipe2 = pickle.load(f)\n",
    "\n",
    "pipe2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (42_004, 19)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>city_category</th><th>employer_category1</th><th>employer_category2</th><th>monthly_income</th><th>existing_emi</th><th>loan_amount</th><th>loan_period</th><th>interest_rate</th><th>emi</th><th>var1</th><th>approved</th><th>test_col</th><th>existing_emi_log1p</th><th>loan_amount_log1p</th><th>loan_amount_log1p_clipped</th><th>loan_amount_sqrt</th><th>loan_amount_lag_1</th><th>employer_category1_is_missing</th><th>gender_Male</th></tr><tr><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>i8</td><td>i8</td><td>str</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>f32</td><td>u8</td><td>u8</td></tr></thead><tbody><tr><td>0.080959</td><td>0.026603</td><td>-3.106548</td><td>-0.017613</td><td>-0.096348</td><td>-0.632338</td><td>-1.619415</td><td>-1.019936</td><td>-0.197259</td><td>10</td><td>0</td><td>&quot;TEST&quot;</td><td>0.0</td><td>-0.586105</td><td>NaN</td><td>-0.658025</td><td>0.181257</td><td>0</td><td>1</td></tr><tr><td>-0.479553</td><td>0.026603</td><td>0.371879</td><td>-0.023167</td><td>-0.096348</td><td>0.181273</td><td>0.093681</td><td>null</td><td>null</td><td>0</td><td>0</td><td>&quot;TEST&quot;</td><td>0.0</td><td>0.537137</td><td>NaN</td><td>0.375948</td><td>1.710834</td><td>0</td><td>1</td></tr><tr><td>-0.479553</td><td>0.014737</td><td>0.371879</td><td>-0.017613</td><td>-0.096348</td><td>1.710861</td><td>0.950229</td><td>null</td><td>null</td><td>7</td><td>0</td><td>&quot;TEST&quot;</td><td>0.0</td><td>1.527696</td><td>NaN</td><td>1.709278</td><td>0.343978</td><td>0</td><td>1</td></tr><tr><td>0.080959</td><td>0.014737</td><td>0.371879</td><td>0.011269</td><td>0.810234</td><td>0.343995</td><td>-1.619415</td><td>null</td><td>null</td><td>10</td><td>0</td><td>&quot;TEST&quot;</td><td>7.824446</td><td>0.683076</td><td>NaN</td><td>0.543738</td><td>2.947513</td><td>0</td><td>1</td></tr><tr><td>-0.046475</td><td>0.024336</td><td>0.371879</td><td>0.000161</td><td>-0.096348</td><td>2.947549</td><td>0.950229</td><td>-0.746292</td><td>2.631433</td><td>10</td><td>0</td><td>&quot;TEST&quot;</td><td>0.0</td><td>2.00661</td><td>NaN</td><td>2.546276</td><td>-0.306906</td><td>0</td><td>1</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>0.080959</td><td>0.024336</td><td>-1.947072</td><td>-0.012725</td><td>-0.096348</td><td>-0.306893</td><td>-0.762867</td><td>-1.062693</td><td>-0.120198</td><td>10</td><td>0</td><td>&quot;TEST&quot;</td><td>0.0</td><td>-0.024488</td><td>NaN</td><td>-0.193265</td><td>-0.306906</td><td>0</td><td>1</td></tr><tr><td>-0.479553</td><td>0.014737</td><td>-3.106548</td><td>-0.022501</td><td>-0.096348</td><td>-0.306893</td><td>-0.762867</td><td>null</td><td>null</td><td>2</td><td>0</td><td>&quot;TEST&quot;</td><td>0.0</td><td>-0.024488</td><td>NaN</td><td>-0.193265</td><td>-0.502171</td><td>0</td><td>1</td></tr><tr><td>-0.046475</td><td>0.026603</td><td>0.371879</td><td>-0.026055</td><td>-0.096348</td><td>-0.50216</td><td>0.093681</td><td>2.785431</td><td>-0.210546</td><td>2</td><td>0</td><td>&quot;TEST&quot;</td><td>0.0</td><td>-0.333569</td><td>NaN</td><td>-0.46065</td><td>1.320303</td><td>0</td><td>0</td></tr><tr><td>0.080959</td><td>0.026603</td><td>-0.787597</td><td>0.010794</td><td>0.399009</td><td>1.320328</td><td>0.950229</td><td>null</td><td>null</td><td>10</td><td>0</td><td>&quot;TEST&quot;</td><td>7.220374</td><td>1.334103</td><td>NaN</td><td>1.40992</td><td>0.962317</td><td>0</td><td>1</td></tr><tr><td>0.080959</td><td>0.014737</td><td>-3.106548</td><td>-0.014369</td><td>-0.096348</td><td>0.962339</td><td>0.093681</td><td>-0.893375</td><td>1.041041</td><td>10</td><td>0</td><td>&quot;TEST&quot;</td><td>0.0</td><td>1.129211</td><td>NaN</td><td>1.115067</td><td>null</td><td>0</td><td>1</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (42_004, 19)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ city_cate ┆ employer_ ┆ employer_ ┆ monthly_i ┆ … ┆ loan_amou ┆ loan_amou ┆ employer_ ┆ gender_M │\n",
       "│ gory      ┆ category1 ┆ category2 ┆ ncome     ┆   ┆ nt_sqrt   ┆ nt_lag_1  ┆ category1 ┆ ale      │\n",
       "│ ---       ┆ ---       ┆ ---       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ _is_missi ┆ ---      │\n",
       "│ f32       ┆ f32       ┆ f32       ┆ f32       ┆   ┆ f32       ┆ f32       ┆ ng        ┆ u8       │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ ---       ┆          │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆ u8        ┆          │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "│ 0.080959  ┆ 0.026603  ┆ -3.106548 ┆ -0.017613 ┆ … ┆ -0.658025 ┆ 0.181257  ┆ 0         ┆ 1        │\n",
       "│ -0.479553 ┆ 0.026603  ┆ 0.371879  ┆ -0.023167 ┆ … ┆ 0.375948  ┆ 1.710834  ┆ 0         ┆ 1        │\n",
       "│ -0.479553 ┆ 0.014737  ┆ 0.371879  ┆ -0.017613 ┆ … ┆ 1.709278  ┆ 0.343978  ┆ 0         ┆ 1        │\n",
       "│ 0.080959  ┆ 0.014737  ┆ 0.371879  ┆ 0.011269  ┆ … ┆ 0.543738  ┆ 2.947513  ┆ 0         ┆ 1        │\n",
       "│ -0.046475 ┆ 0.024336  ┆ 0.371879  ┆ 0.000161  ┆ … ┆ 2.546276  ┆ -0.306906 ┆ 0         ┆ 1        │\n",
       "│ …         ┆ …         ┆ …         ┆ …         ┆ … ┆ …         ┆ …         ┆ …         ┆ …        │\n",
       "│ 0.080959  ┆ 0.024336  ┆ -1.947072 ┆ -0.012725 ┆ … ┆ -0.193265 ┆ -0.306906 ┆ 0         ┆ 1        │\n",
       "│ -0.479553 ┆ 0.014737  ┆ -3.106548 ┆ -0.022501 ┆ … ┆ -0.193265 ┆ -0.502171 ┆ 0         ┆ 1        │\n",
       "│ -0.046475 ┆ 0.026603  ┆ 0.371879  ┆ -0.026055 ┆ … ┆ -0.46065  ┆ 1.320303  ┆ 0         ┆ 0        │\n",
       "│ 0.080959  ┆ 0.026603  ┆ -0.787597 ┆ 0.010794  ┆ … ┆ 1.40992   ┆ 0.962317  ┆ 0         ┆ 1        │\n",
       "│ 0.080959  ┆ 0.014737  ┆ -3.106548 ┆ -0.014369 ┆ … ┆ 1.115067  ┆ null      ┆ 0         ┆ 1        │\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_transformed_2 = pipe2.transform(df)\n",
    "df_transformed_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from polars.testing import assert_frame_equal\n",
    "# True\n",
    "assert_frame_equal(df_transformed, df_transformed_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'example',\n",
       " 'target': 'approved',\n",
       " 'feature_names_in_': ['ID',\n",
       "  'Gender',\n",
       "  'DOB',\n",
       "  'Lead_Creation_Date',\n",
       "  'City_Code',\n",
       "  'City_Category',\n",
       "  'Employer_Code',\n",
       "  'Employer_Category1',\n",
       "  'Employer_Category2',\n",
       "  'Monthly_Income',\n",
       "  'Customer_Existing_Primary_Bank_Code',\n",
       "  'Primary_Bank_Type',\n",
       "  'Contacted',\n",
       "  'Source',\n",
       "  'Source_Category',\n",
       "  'Existing_EMI',\n",
       "  'Loan_Amount',\n",
       "  'Loan_Period',\n",
       "  'Interest_Rate',\n",
       "  'EMI',\n",
       "  'Var1',\n",
       "  'Approved'],\n",
       " 'feature_names_out_': ['city_category',\n",
       "  'employer_category1',\n",
       "  'employer_category2',\n",
       "  'monthly_income',\n",
       "  'existing_emi',\n",
       "  'loan_amount',\n",
       "  'loan_period',\n",
       "  'interest_rate',\n",
       "  'emi',\n",
       "  'var1',\n",
       "  'approved',\n",
       "  'test_col',\n",
       "  'existing_emi_log1p',\n",
       "  'loan_amount_log1p',\n",
       "  'loan_amount_log1p_clipped',\n",
       "  'loan_amount_sqrt',\n",
       "  'loan_amount_lag_1',\n",
       "  'employer_category1_is_missing',\n",
       "  'gender_Male'],\n",
       " 'transforms': [{'SelectStep': ['{\"Alias\":[{\"Column\":\"ID\"},\"id\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Gender\"},\"gender\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"DOB\"},\"dob\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Lead_Creation_Date\"},\"lead_creation_date\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"City_Code\"},\"city_code\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"City_Category\"},\"city_category\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Employer_Code\"},\"employer_code\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Employer_Category1\"},\"employer_category1\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Employer_Category2\"},\"employer_category2\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Monthly_Income\"},\"monthly_income\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Customer_Existing_Primary_Bank_Code\"},\"customer_existing_primary_bank_code\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Primary_Bank_Type\"},\"primary_bank_type\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Contacted\"},\"contacted\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Source\"},\"source\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Source_Category\"},\"source_category\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Existing_EMI\"},\"existing_emi\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Loan_Amount\"},\"loan_amount\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Loan_Period\"},\"loan_period\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Interest_Rate\"},\"interest_rate\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"EMI\"},\"emi\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Var1\"},\"var1\"]}',\n",
       "    '{\"Alias\":[{\"Column\":\"Approved\"},\"approved\"]}']},\n",
       "  {'SQLStep': \"\\nselect\\n*\\n, 'TEST' as test_col\\nfrom df\\nwhere loan_amount is not null\\n\"},\n",
       "  {'FilterStep': ['{\"Function\":{\"input\":[{\"Function\":{\"input\":[{\"Column\":\"city_category\"}],\"function\":{\"Boolean\":\"IsNotNull\"},\"options\":{\"collect_groups\":\"ElementWise\",\"fmt_str\":\"\",\"check_lengths\":true,\"flags\":\"ALLOW_GROUP_AWARE\"}}}],\"function\":{\"Boolean\":\"AllHorizontal\"},\"options\":{\"collect_groups\":\"GroupWise\",\"fmt_str\":\"\",\"check_lengths\":true,\"flags\":\"ALLOW_GROUP_AWARE | INPUT_WILDCARD_EXPANSION | ALLOW_EMPTY_INPUTS\"}}}']},\n",
       "  {'SelectStep': ['{\"Selector\":{\"Add\":[{\"Root\":{\"DtypeColumn\":[{\"Decimal\":[null,null]},\"Float32\",\"UInt32\",\"UInt8\",\"Int32\",\"Int8\",\"Float64\",\"UInt64\",\"UInt16\",\"Int64\",\"Int16\"]}},{\"Root\":{\"Columns\":[\"gender\",\"employer_category1\",\"city_category\",\"test_col\"]}}]}}']},\n",
       "  {'WithColumnsStep': ['{\"Alias\":[{\"Ternary\":{\"predicate\":{\"Function\":{\"input\":[{\"Column\":\"loan_period\"}],\"function\":{\"Boolean\":\"IsNull\"},\"options\":{\"collect_groups\":\"ElementWise\",\"fmt_str\":\"\",\"check_lengths\":true,\"flags\":\"ALLOW_GROUP_AWARE\"}}},\"truthy\":{\"Function\":{\"input\":[{\"BinaryExpr\":{\"left\":{\"Column\":\"var1\"},\"op\":\"Multiply\",\"right\":{\"Literal\":{\"Float\":0.5098100117596632}}}},{\"BinaryExpr\":{\"left\":{\"Column\":\"existing_emi\"},\"op\":\"Multiply\",\"right\":{\"Literal\":{\"Float\":-7.604079653752158e-6}}}}],\"function\":\"SumHorizontal\",\"options\":{\"collect_groups\":\"ElementWise\",\"fmt_str\":\"\",\"check_lengths\":true,\"flags\":\"ALLOW_GROUP_AWARE | INPUT_WILDCARD_EXPANSION\"}}},\"falsy\":{\"Cast\":{\"expr\":{\"Column\":\"loan_period\"},\"dtype\":\"Float64\",\"options\":\"Strict\"}}}},\"loan_period\"]}']},\n",
       "  {'WithColumnsStep': ['{\"Function\":{\"input\":[{\"Column\":\"existing_emi\"},{\"Literal\":{\"Float\":0.0}}],\"function\":\"FillNull\",\"options\":{\"collect_groups\":\"ElementWise\",\"fmt_str\":\"\",\"check_lengths\":true,\"flags\":\"ALLOW_GROUP_AWARE\"}}}']},\n",
       "  {'WithColumnsStep': ['{\"Alias\":[{\"Function\":{\"input\":[{\"Column\":\"existing_emi\"}],\"function\":\"Log1p\",\"options\":{\"collect_groups\":\"ElementWise\",\"fmt_str\":\"\",\"check_lengths\":true,\"flags\":\"ALLOW_GROUP_AWARE\"}}},\"existing_emi_log1p\"]}',\n",
       "    '{\"Alias\":[{\"Function\":{\"input\":[{\"Column\":\"loan_amount\"}],\"function\":\"Log1p\",\"options\":{\"collect_groups\":\"ElementWise\",\"fmt_str\":\"\",\"check_lengths\":true,\"flags\":\"ALLOW_GROUP_AWARE\"}}},\"loan_amount_log1p\"]}',\n",
       "    '{\"Alias\":[{\"Function\":{\"input\":[{\"Column\":\"loan_amount\"},{\"Literal\":{\"Int\":0}},{\"Literal\":{\"Int\":1000}}],\"function\":{\"Clip\":{\"has_min\":true,\"has_max\":true}},\"options\":{\"collect_groups\":\"ElementWise\",\"fmt_str\":\"\",\"check_lengths\":true,\"flags\":\"ALLOW_GROUP_AWARE\"}}},\"loan_amount_log1p_clipped\"]}',\n",
       "    '{\"Alias\":[{\"Function\":{\"input\":[{\"Column\":\"loan_amount\"}],\"function\":{\"Pow\":\"Sqrt\"},\"options\":{\"collect_groups\":\"ElementWise\",\"fmt_str\":\"\",\"check_lengths\":true,\"flags\":\"ALLOW_GROUP_AWARE\"}}},\"loan_amount_sqrt\"]}',\n",
       "    '{\"Alias\":[{\"Function\":{\"input\":[{\"Column\":\"loan_amount\"},{\"Literal\":{\"Int\":-1}}],\"function\":\"Shift\",\"options\":{\"collect_groups\":\"GroupWise\",\"fmt_str\":\"\",\"check_lengths\":true,\"flags\":\"ALLOW_GROUP_AWARE\"}}},\"loan_amount_lag_1\"]}']},\n",
       "  {'WithColumnsStep': ['{\"BinaryExpr\":{\"left\":{\"BinaryExpr\":{\"left\":{\"Column\":\"employer_category2\"},\"op\":\"Minus\",\"right\":{\"Literal\":{\"Float\":3.679269695227142}}}},\"op\":\"TrueDivide\",\"right\":{\"Literal\":{\"Float\":0.8624587860675418}}}}',\n",
       "    '{\"BinaryExpr\":{\"left\":{\"BinaryExpr\":{\"left\":{\"Column\":\"monthly_income\"},\"op\":\"Minus\",\"right\":{\"Literal\":{\"Float\":7463.797309780022}}}},\"op\":\"TrueDivide\",\"right\":{\"Literal\":{\"Float\":225051.544361042}}}}',\n",
       "    '{\"BinaryExpr\":{\"left\":{\"BinaryExpr\":{\"left\":{\"Column\":\"existing_emi\"},\"op\":\"Minus\",\"right\":{\"Literal\":{\"Float\":265.69024752404533}}}},\"op\":\"TrueDivide\",\"right\":{\"Literal\":{\"Float\":2757.6092535058197}}}}',\n",
       "    '{\"BinaryExpr\":{\"left\":{\"BinaryExpr\":{\"left\":{\"Column\":\"loan_amount\"},\"op\":\"Minus\",\"right\":{\"Literal\":{\"Float\":39429.98285877536}}}},\"op\":\"TrueDivide\",\"right\":{\"Literal\":{\"Float\":30727.230218132237}}}}',\n",
       "    '{\"BinaryExpr\":{\"left\":{\"BinaryExpr\":{\"left\":{\"Column\":\"loan_period\"},\"op\":\"Minus\",\"right\":{\"Literal\":{\"Float\":3.8906294638605847}}}},\"op\":\"TrueDivide\",\"right\":{\"Literal\":{\"Float\":1.1674769269411276}}}}',\n",
       "    '{\"BinaryExpr\":{\"left\":{\"BinaryExpr\":{\"left\":{\"Column\":\"interest_rate\"},\"op\":\"Minus\",\"right\":{\"Literal\":{\"Float\":19.21356976117795}}}},\"op\":\"TrueDivide\",\"right\":{\"Literal\":{\"Float\":5.8470045315482775}}}}',\n",
       "    '{\"BinaryExpr\":{\"left\":{\"BinaryExpr\":{\"left\":{\"Column\":\"emi\"},\"op\":\"Minus\",\"right\":{\"Literal\":{\"Float\":1101.4662416950978}}}},\"op\":\"TrueDivide\",\"right\":{\"Literal\":{\"Float\":752.644499525087}}}}',\n",
       "    '{\"BinaryExpr\":{\"left\":{\"BinaryExpr\":{\"left\":{\"Column\":\"loan_amount_log1p\"},\"op\":\"Minus\",\"right\":{\"Literal\":{\"Float\":10.326664500589034}}}},\"op\":\"TrueDivide\",\"right\":{\"Literal\":{\"Float\":0.7219306388560957}}}}',\n",
       "    '{\"BinaryExpr\":{\"left\":{\"BinaryExpr\":{\"left\":{\"Column\":\"loan_amount_log1p_clipped\"},\"op\":\"Minus\",\"right\":{\"Literal\":{\"Float\":1000.0}}}},\"op\":\"TrueDivide\",\"right\":{\"Literal\":{\"Float\":0.0}}}}',\n",
       "    '{\"BinaryExpr\":{\"left\":{\"BinaryExpr\":{\"left\":{\"Column\":\"loan_amount_sqrt\"},\"op\":\"Minus\",\"right\":{\"Literal\":{\"Float\":186.42196063934287}}}},\"op\":\"TrueDivide\",\"right\":{\"Literal\":{\"Float\":68.38739247959863}}}}',\n",
       "    '{\"BinaryExpr\":{\"left\":{\"BinaryExpr\":{\"left\":{\"Column\":\"loan_amount_lag_1\"},\"op\":\"Minus\",\"right\":{\"Literal\":{\"Float\":39430.44544437302}}}},\"op\":\"TrueDivide\",\"right\":{\"Literal\":{\"Float\":30727.449732966954}}}}']},\n",
       "  {'WithColumnsStep': ['{\"Alias\":[{\"Cast\":{\"expr\":{\"Function\":{\"input\":[{\"Column\":\"employer_category1\"}],\"function\":{\"Boolean\":\"IsNull\"},\"options\":{\"collect_groups\":\"ElementWise\",\"fmt_str\":\"\",\"check_lengths\":true,\"flags\":\"ALLOW_GROUP_AWARE\"}}},\"dtype\":\"UInt8\",\"options\":\"Strict\"}},\"employer_category1_is_missing\"]}']},\n",
       "  {'WithColumnsStep': ['{\"Alias\":[{\"Cast\":{\"expr\":{\"BinaryExpr\":{\"left\":{\"Column\":\"gender\"},\"op\":\"EqValidity\",\"right\":{\"Literal\":{\"String\":\"Male\"}}}},\"dtype\":\"UInt8\",\"options\":\"Strict\"}},\"gender_Male\"]}']},\n",
       "  {'SelectStep': ['{\"Exclude\":[\"Wildcard\",[{\"Name\":\"gender\"}]]}']},\n",
       "  {'WithColumnsStep': ['{\"Function\":{\"input\":[{\"Column\":\"city_category\"},{\"Literal\":{\"Series\":{\"name\":\"value\",\"datatype\":\"String\",\"bit_settings\":\"\",\"values\":[\"B\",\"A\",\"C\"]}}},{\"Literal\":{\"Series\":{\"name\":\"woe\",\"datatype\":\"Float64\",\"bit_settings\":\"\",\"values\":[-0.04647519483535344,0.0809586180645928,-0.47955283435510176]}}},{\"Literal\":\"Null\"}],\"function\":{\"ReplaceStrict\":{\"return_dtype\":null}},\"options\":{\"collect_groups\":\"ElementWise\",\"fmt_str\":\"\",\"check_lengths\":true,\"flags\":\"ALLOW_GROUP_AWARE\"}}}']},\n",
       "  {'WithColumnsStep': ['{\"Function\":{\"input\":[{\"Column\":\"employer_category1\"},{\"Literal\":{\"Series\":{\"name\":\"value\",\"datatype\":\"String\",\"bit_settings\":\"\",\"values\":[\"A\",\"C\",\"B\"]}}},{\"Literal\":{\"Series\":{\"name\":\"to\",\"datatype\":\"Float64\",\"bit_settings\":\"\",\"values\":[0.014736842105263158,0.02660307366189719,0.024335548172757474]}}},{\"Literal\":\"Null\"}],\"function\":{\"ReplaceStrict\":{\"return_dtype\":null}},\"options\":{\"collect_groups\":\"ElementWise\",\"fmt_str\":\"\",\"check_lengths\":true,\"flags\":\"ALLOW_GROUP_AWARE\"}}}']},\n",
       "  {'WithColumnsStep': ['{\"Function\":{\"input\":[{\"DtypeColumn\":[\"UInt32\",\"UInt8\",\"Int32\",\"Int8\",\"UInt64\",\"UInt16\",\"Int64\",\"Int16\"]}],\"function\":\"ShrinkType\",\"options\":{\"collect_groups\":\"GroupWise\",\"fmt_str\":\"\",\"check_lengths\":true,\"flags\":\"ALLOW_GROUP_AWARE\"}}}']},\n",
       "  {'WithColumnsStep': ['{\"Cast\":{\"expr\":{\"DtypeColumn\":[\"Float32\",\"Float64\"]},\"dtype\":\"Float32\",\"options\":\"Strict\"}}']}],\n",
       " 'ensure_features_in': False,\n",
       " 'ensure_features_out': True}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To save the pipeline as JSON\n",
    "\n",
    "pipe.to_json(\"test.json\")\n",
    "pipe3 = Pipeline.from_json(\"test.json\")\n",
    "# True\n",
    "assert_frame_equal(df_transformed, pipe3.transform(df))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Custom Tranformations in Pipeline\n",
    "\n",
    "Need version >= v0.4.6 (Not released yet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 22)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>ID</th><th>Gender</th><th>DOB</th><th>Lead_Creation_Date</th><th>City_Code</th><th>City_Category</th><th>Employer_Code</th><th>Employer_Category1</th><th>Employer_Category2</th><th>Monthly_Income</th><th>Customer_Existing_Primary_Bank_Code</th><th>Primary_Bank_Type</th><th>Contacted</th><th>Source</th><th>Source_Category</th><th>Existing_EMI</th><th>Loan_Amount</th><th>Loan_Period</th><th>Interest_Rate</th><th>EMI</th><th>Var1</th><th>Approved</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>i64</td><td>f64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>i64</td><td>i64</td><td>f64</td><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>&quot;APPC90493171225&quot;</td><td>&quot;Female&quot;</td><td>&quot;23/07/79&quot;</td><td>&quot;15/07/16&quot;</td><td>&quot;C10001&quot;</td><td>&quot;A&quot;</td><td>&quot;COM0044082&quot;</td><td>&quot;A&quot;</td><td>4</td><td>2000.0</td><td>&quot;B001&quot;</td><td>&quot;P&quot;</td><td>&quot;N&quot;</td><td>&quot;S122&quot;</td><td>&quot;G&quot;</td><td>0.0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td></tr><tr><td>&quot;APPD40611263344&quot;</td><td>&quot;Male&quot;</td><td>&quot;07/12/86&quot;</td><td>&quot;04/07/16&quot;</td><td>&quot;C10003&quot;</td><td>&quot;A&quot;</td><td>&quot;COM0000002&quot;</td><td>&quot;C&quot;</td><td>1</td><td>3500.0</td><td>&quot;B002&quot;</td><td>&quot;P&quot;</td><td>&quot;Y&quot;</td><td>&quot;S122&quot;</td><td>&quot;G&quot;</td><td>0.0</td><td>20000</td><td>2</td><td>13.25</td><td>953</td><td>10</td><td>0</td></tr><tr><td>&quot;APPE70289249423&quot;</td><td>&quot;Male&quot;</td><td>&quot;10/12/82&quot;</td><td>&quot;19/07/16&quot;</td><td>&quot;C10125&quot;</td><td>&quot;C&quot;</td><td>&quot;COM0005267&quot;</td><td>&quot;C&quot;</td><td>4</td><td>2250.0</td><td>&quot;B003&quot;</td><td>&quot;G&quot;</td><td>&quot;Y&quot;</td><td>&quot;S143&quot;</td><td>&quot;B&quot;</td><td>0.0</td><td>45000</td><td>4</td><td>null</td><td>null</td><td>0</td><td>0</td></tr><tr><td>&quot;APPF80273865537&quot;</td><td>&quot;Male&quot;</td><td>&quot;30/01/89&quot;</td><td>&quot;09/07/16&quot;</td><td>&quot;C10477&quot;</td><td>&quot;C&quot;</td><td>&quot;COM0004143&quot;</td><td>&quot;A&quot;</td><td>4</td><td>3500.0</td><td>&quot;B003&quot;</td><td>&quot;G&quot;</td><td>&quot;Y&quot;</td><td>&quot;S143&quot;</td><td>&quot;B&quot;</td><td>0.0</td><td>92000</td><td>5</td><td>null</td><td>null</td><td>7</td><td>0</td></tr><tr><td>&quot;APPG60994436641&quot;</td><td>&quot;Male&quot;</td><td>&quot;19/04/85&quot;</td><td>&quot;20/07/16&quot;</td><td>&quot;C10002&quot;</td><td>&quot;A&quot;</td><td>&quot;COM0001781&quot;</td><td>&quot;A&quot;</td><td>4</td><td>10000.0</td><td>&quot;B001&quot;</td><td>&quot;P&quot;</td><td>&quot;Y&quot;</td><td>&quot;S134&quot;</td><td>&quot;B&quot;</td><td>2500.0</td><td>50000</td><td>2</td><td>null</td><td>null</td><td>10</td><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 22)\n",
       "┌────────────────┬────────┬──────────┬────────────────┬───┬───────────────┬──────┬──────┬──────────┐\n",
       "│ ID             ┆ Gender ┆ DOB      ┆ Lead_Creation_ ┆ … ┆ Interest_Rate ┆ EMI  ┆ Var1 ┆ Approved │\n",
       "│ ---            ┆ ---    ┆ ---      ┆ Date           ┆   ┆ ---           ┆ ---  ┆ ---  ┆ ---      │\n",
       "│ str            ┆ str    ┆ str      ┆ ---            ┆   ┆ f64           ┆ i64  ┆ i64  ┆ i64      │\n",
       "│                ┆        ┆          ┆ str            ┆   ┆               ┆      ┆      ┆          │\n",
       "╞════════════════╪════════╪══════════╪════════════════╪═══╪═══════════════╪══════╪══════╪══════════╡\n",
       "│ APPC9049317122 ┆ Female ┆ 23/07/79 ┆ 15/07/16       ┆ … ┆ null          ┆ null ┆ 0    ┆ 0        │\n",
       "│ 5              ┆        ┆          ┆                ┆   ┆               ┆      ┆      ┆          │\n",
       "│ APPD4061126334 ┆ Male   ┆ 07/12/86 ┆ 04/07/16       ┆ … ┆ 13.25         ┆ 953  ┆ 10   ┆ 0        │\n",
       "│ 4              ┆        ┆          ┆                ┆   ┆               ┆      ┆      ┆          │\n",
       "│ APPE7028924942 ┆ Male   ┆ 10/12/82 ┆ 19/07/16       ┆ … ┆ null          ┆ null ┆ 0    ┆ 0        │\n",
       "│ 3              ┆        ┆          ┆                ┆   ┆               ┆      ┆      ┆          │\n",
       "│ APPF8027386553 ┆ Male   ┆ 30/01/89 ┆ 09/07/16       ┆ … ┆ null          ┆ null ┆ 7    ┆ 0        │\n",
       "│ 7              ┆        ┆          ┆                ┆   ┆               ┆      ┆      ┆          │\n",
       "│ APPG6099443664 ┆ Male   ┆ 19/04/85 ┆ 20/07/16       ┆ … ┆ null          ┆ null ┆ 10   ┆ 0        │\n",
       "│ 1              ┆        ┆          ┆                ┆   ┆               ┆      ┆      ┆          │\n",
       "└────────────────┴────────┴──────────┴────────────────┴───┴───────────────┴──────┴──────┴──────────┘"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pl.read_parquet(\"../examples/dependency.parquet\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Union, List\n",
    "\n",
    "# Any custom function must satistfy the following function signature:\n",
    "# func(df:Union[pl.DataFrame, pl.LazyFrame], cols: List[str], ...) -> List[pl.Expr]\n",
    "# where ... means kwargs\n",
    "# Here is a custom imputer\n",
    "\n",
    "def smallest_abs_impute(df:Union[pl.DataFrame, pl.LazyFrame], cols: List[str], epsilon:float = 0.01) -> List[pl.Expr]:\n",
    "    \"\"\"\n",
    "    Imputes columns by the min of the absolute values for c in columns, plus epsilon.\n",
    "    \"\"\"\n",
    "    temp = df.lazy().select(pl.col(cols).abs().min() + epsilon).collect().row(0)\n",
    "    return [pl.col(c).fill_null(m) for c, m in zip(cols, temp)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Naive Query Steps: \n",
       "\n",
       "Step 1:\n",
       "col(\"ID\").alias(\"id\"),\n",
       "col(\"Gender\").alias(\"gender\"),\n",
       "col(\"DOB\").alias(\"dob\"),\n",
       "col(\"Lead_Creation_Date\").alias(\"lead_creation_date\"),\n",
       "col(\"City_Code\").alias(\"city_code\"),\n",
       "col(\"City_Category\").alias(\"city_category\"),\n",
       "col(\"Employer_Code\").alias(\"employer_code\"),\n",
       "col(\"Employer_Category1\").alias(\"employer_category1\"),\n",
       "col(\"Employer_Category2\").alias(\"employer_category2\"),\n",
       "col(\"Monthly_Income\").alias(\"monthly_income\"),\n",
       "col(\"Customer_Existing_Primary_Bank_Code\").alias(\"customer_existing_primary_bank_code\"),\n",
       "col(\"Primary_Bank_Type\").alias(\"primary_bank_type\"),\n",
       "col(\"Contacted\").alias(\"contacted\"),\n",
       "col(\"Source\").alias(\"source\"),\n",
       "col(\"Source_Category\").alias(\"source_category\"),\n",
       "col(\"Existing_EMI\").alias(\"existing_emi\"),\n",
       "col(\"Loan_Amount\").alias(\"loan_amount\"),\n",
       "col(\"Loan_Period\").alias(\"loan_period\"),\n",
       "col(\"Interest_Rate\").alias(\"interest_rate\"),\n",
       "col(\"EMI\").alias(\"emi\"),\n",
       "col(\"Var1\").alias(\"var1\"),\n",
       "col(\"Approved\").alias(\"approved\")\n",
       "\n",
       "Step 2:\n",
       "col(\"var1\").fill_null([dyn float: 0.5]),\n",
       "col(\"existing_emi\").fill_null([dyn float: 0.5]),\n",
       "col(\"loan_amount\").fill_null([dyn float: 5000.5])\n"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bp = (\n",
    "    Blueprint(df, name = \"example\", target = \"approved\")\n",
    "    .lowercase() # lowercase all columns\n",
    "    .append_fit_func(smallest_abs_impute, [\"var1\", \"existing_emi\", \"loan_amount\"], epsilon = 0.5)\n",
    "    # Use append_fit_func for custom transforms\n",
    ")\n",
    "# Notice that the value to impute is correct, it is 0.5, because the min abs of the columns are 0.\n",
    "pipe:Pipeline = bp.materialize()\n",
    "pipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>var1</th><th>existing_emi</th><th>loan_amount</th></tr><tr><td>u32</td><td>u32</td><td>u32</td></tr></thead><tbody><tr><td>0</td><td>0</td><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 3)\n",
       "┌──────┬──────────────┬─────────────┐\n",
       "│ var1 ┆ existing_emi ┆ loan_amount │\n",
       "│ ---  ┆ ---          ┆ ---         │\n",
       "│ u32  ┆ u32          ┆ u32         │\n",
       "╞══════╪══════════════╪═════════════╡\n",
       "│ 0    ┆ 0            ┆ 0           │\n",
       "└──────┴──────────────┴─────────────┘"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe.transform(df).null_count().select([\"var1\", \"existing_emi\", \"loan_amount\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scriptable Steps in Pipeline\n",
    "\n",
    "What is a scriptable step? It means we can encode steps in a json or yaml file easily. As long as we can turn the text into \n",
    "a valid Python dictionary, the step (the transformation) can be defined."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Existing_EMI</th></tr><tr><td>u32</td></tr></thead><tbody><tr><td>51</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 1)\n",
       "┌──────────────┐\n",
       "│ Existing_EMI │\n",
       "│ ---          │\n",
       "│ u32          │\n",
       "╞══════════════╡\n",
       "│ 51           │\n",
       "└──────────────┘"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select(\n",
    "    pl.col(\"Existing_EMI\").null_count() \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Existing_EMI</th></tr><tr><td>u32</td></tr></thead><tbody><tr><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 1)\n",
       "┌──────────────┐\n",
       "│ Existing_EMI │\n",
       "│ ---          │\n",
       "│ u32          │\n",
       "╞══════════════╡\n",
       "│ 0            │\n",
       "└──────────────┘"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bp = Blueprint(df, name = \"example\", target = \"approved\")\n",
    "\n",
    "# Takes in a dict with 3 fields: `name`, `args`, and `kwargs`. \n",
    "# Args and kwargs are optional depending on whether the method call needs certain arguments.\n",
    "step_dict_1 = {\n",
    "    \"name\": \"impute\",\n",
    "    \"kwargs\": {\"cols\": [\"Existing_EMI\"], \"method\": \"median\"} \n",
    "}\n",
    "\n",
    "step_dict_2 = {\n",
    "    \"name\": \"does_not_exist\",\n",
    "    \"kwargs\": {\"test\": 1} \n",
    "}\n",
    "\n",
    "# filter_step = {\n",
    "#     \"name\": \"filter\",\n",
    "#     \"args\": [\"Employer_Category1 is not null\"]\n",
    "# }\n",
    "\n",
    "bp.append_step_from_dict(\n",
    "    step_dict_1\n",
    ")\n",
    "\n",
    "# .append_step_from_dict(\n",
    "#     filter_step\n",
    "# )\n",
    "\n",
    "# bp.append_step_from_dict(step_dict_2) # Will error\n",
    "pipe = bp.materialize()\n",
    "# \n",
    "df_transformed = pipe.transform(df)\n",
    "df_transformed.select(\n",
    "    pl.col(\"Existing_EMI\").null_count() # Imputed. So 0\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Custom Transforms as a Scriptable Step in Pipeline\n",
    "\n",
    "You need to inherit the blueprint class. Once the blueprint is materialized (learned). You do not need this class any more, because the \"learned\" info should all be encoded as Polars expressions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from polars_ds.pipeline import Blueprint, FitStep\n",
    "from typing import Self, Union, List\n",
    "from functools import partial\n",
    "\n",
    "def smallest_abs_impute(df:Union[pl.DataFrame, pl.LazyFrame], cols: List[str], epsilon:float = 0.01) -> List[pl.Expr]:\n",
    "    \"\"\"\n",
    "    Imputes columns by the min of the absolute values for c in columns, plus epsilon.\n",
    "    \"\"\"\n",
    "    temp = df.lazy().select(pl.col(cols).abs().min() + epsilon).collect().row(0)\n",
    "    return [pl.col(c).fill_null(m).name.suffix(\"_imputed\") for c, m in zip(cols, temp)]\n",
    "\n",
    "class ExtendedBlueprint(Blueprint):\n",
    "\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        super().__init__(*args, **kwargs)\n",
    "\n",
    "    def smallest_abs_impute(self, cols: List[str], epsilon:float = 0.01) -> Self:\n",
    "        # bind all arguments, except df and cols.\n",
    "        # If you don't want to use partial from functool, you can define an inner function\n",
    "        partial_func = partial(smallest_abs_impute, epsilon=epsilon)\n",
    "        self._steps.append(\n",
    "            FitStep(partial_func, cols, self.exclude)\n",
    "        )\n",
    "        return self\n",
    "\n",
    "    def smallest_abs_impute2(self, cols: List[str], epsilon:float = 0.01) -> Self:\n",
    "        # bind all arguments, except df and cols.\n",
    "        # Example of using an inner function\n",
    "        def inner_func(df:Union[pl.DataFrame, pl.LazyFrame], cols: List[str]) -> List[pl.Expr]:\n",
    "            temp = df.lazy().select(pl.col(cols).abs().min() + epsilon).collect().row(0)\n",
    "            return [pl.col(c).fill_null(m).name.suffix(\"_imputed2\") for c, m in zip(cols, temp)]\n",
    "\n",
    "        self._steps.append(\n",
    "            FitStep(inner_func, cols, self.exclude)\n",
    "        )\n",
    "        return self\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (51, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>Existing_EMI</th><th>Existing_EMI_imputed</th><th>Existing_EMI_imputed2</th><th>impute_value</th></tr><tr><td>f64</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>null</td><td>0.01</td><td>0.01</td><td>0.01</td></tr><tr><td>null</td><td>0.01</td><td>0.01</td><td>0.01</td></tr><tr><td>null</td><td>0.01</td><td>0.01</td><td>0.01</td></tr><tr><td>null</td><td>0.01</td><td>0.01</td><td>0.01</td></tr><tr><td>null</td><td>0.01</td><td>0.01</td><td>0.01</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>null</td><td>0.01</td><td>0.01</td><td>0.01</td></tr><tr><td>null</td><td>0.01</td><td>0.01</td><td>0.01</td></tr><tr><td>null</td><td>0.01</td><td>0.01</td><td>0.01</td></tr><tr><td>null</td><td>0.01</td><td>0.01</td><td>0.01</td></tr><tr><td>null</td><td>0.01</td><td>0.01</td><td>0.01</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (51, 4)\n",
       "┌──────────────┬──────────────────────┬───────────────────────┬──────────────┐\n",
       "│ Existing_EMI ┆ Existing_EMI_imputed ┆ Existing_EMI_imputed2 ┆ impute_value │\n",
       "│ ---          ┆ ---                  ┆ ---                   ┆ ---          │\n",
       "│ f64          ┆ f64                  ┆ f64                   ┆ f64          │\n",
       "╞══════════════╪══════════════════════╪═══════════════════════╪══════════════╡\n",
       "│ null         ┆ 0.01                 ┆ 0.01                  ┆ 0.01         │\n",
       "│ null         ┆ 0.01                 ┆ 0.01                  ┆ 0.01         │\n",
       "│ null         ┆ 0.01                 ┆ 0.01                  ┆ 0.01         │\n",
       "│ null         ┆ 0.01                 ┆ 0.01                  ┆ 0.01         │\n",
       "│ null         ┆ 0.01                 ┆ 0.01                  ┆ 0.01         │\n",
       "│ …            ┆ …                    ┆ …                     ┆ …            │\n",
       "│ null         ┆ 0.01                 ┆ 0.01                  ┆ 0.01         │\n",
       "│ null         ┆ 0.01                 ┆ 0.01                  ┆ 0.01         │\n",
       "│ null         ┆ 0.01                 ┆ 0.01                  ┆ 0.01         │\n",
       "│ null         ┆ 0.01                 ┆ 0.01                  ┆ 0.01         │\n",
       "│ null         ┆ 0.01                 ┆ 0.01                  ┆ 0.01         │\n",
       "└──────────────┴──────────────────────┴───────────────────────┴──────────────┘"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bp = ExtendedBlueprint(df, name = \"example\", target = \"approved\")\n",
    "\n",
    "# Takes in a dict with 3 fields: `name`, `args`, and `kwargs`. \n",
    "# Args and kwargs are optional depending on whether the method call needs certain arguments.\n",
    "step_dict_1 = {\n",
    "    \"name\": \"smallest_abs_impute\",\n",
    "    \"kwargs\": {\"cols\": [\"Existing_EMI\"], \"epsilon\": 0.01} \n",
    "}\n",
    "\n",
    "step_dict_2 = {\n",
    "    \"name\": \"smallest_abs_impute2\",\n",
    "    \"kwargs\": {\"cols\": [\"Existing_EMI\"], \"epsilon\": 0.01} \n",
    "}\n",
    "\n",
    "bp.append_step_from_dict(\n",
    "    step_dict_1\n",
    ").append_step_from_dict(\n",
    "    step_dict_2\n",
    ")\n",
    "\n",
    "pipe = bp.materialize()\n",
    "df_transformed = pipe.transform(df)\n",
    "\n",
    "df_transformed.with_columns(\n",
    "    impute_value = pl.col(\"Existing_EMI\").abs().min() + 0.01\n",
    ").filter(\n",
    "    pl.col(\"Existing_EMI\").is_null()\n",
    ").select(\n",
    "    pl.col(\"Existing_EMI\"),\n",
    "    pl.col(\"Existing_EMI_imputed\"),\n",
    "    pl.col(\"Existing_EMI_imputed2\"),\n",
    "    pl.col(\"impute_value\")\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Wrapping PDS Pipeline (blueprint) Inside a Sklearn Pipeline\n",
    "\n",
    "It is not recommended, but it is possible. If there is a sklearn pipeline transform you want, and that is not implemented in PDS, please submit a feature request."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (68_899, 12)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>gender</th><th>city_category</th><th>employer_category1</th><th>employer_category2</th><th>monthly_income</th><th>existing_emi</th><th>loan_amount</th><th>loan_period</th><th>interest_rate</th><th>emi</th><th>var1</th><th>approved</th></tr><tr><td>str</td><td>str</td><td>str</td><td>i64</td><td>f64</td><td>f64</td><td>i64</td><td>f64</td><td>f64</td><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>&quot;Female&quot;</td><td>&quot;A&quot;</td><td>&quot;A&quot;</td><td>4</td><td>2000.0</td><td>0.0</td><td>null</td><td>0.0</td><td>null</td><td>null</td><td>0</td><td>0</td></tr><tr><td>&quot;Male&quot;</td><td>&quot;A&quot;</td><td>&quot;C&quot;</td><td>1</td><td>3500.0</td><td>0.0</td><td>20000</td><td>2.0</td><td>13.25</td><td>953</td><td>10</td><td>0</td></tr><tr><td>&quot;Male&quot;</td><td>&quot;C&quot;</td><td>&quot;C&quot;</td><td>4</td><td>2250.0</td><td>0.0</td><td>45000</td><td>4.0</td><td>null</td><td>null</td><td>0</td><td>0</td></tr><tr><td>&quot;Male&quot;</td><td>&quot;C&quot;</td><td>&quot;A&quot;</td><td>4</td><td>3500.0</td><td>0.0</td><td>92000</td><td>5.0</td><td>null</td><td>null</td><td>7</td><td>0</td></tr><tr><td>&quot;Male&quot;</td><td>&quot;A&quot;</td><td>&quot;A&quot;</td><td>4</td><td>10000.0</td><td>2500.0</td><td>50000</td><td>2.0</td><td>null</td><td>null</td><td>10</td><td>0</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;Female&quot;</td><td>&quot;A&quot;</td><td>&quot;A&quot;</td><td>1</td><td>4900.0</td><td>0.0</td><td>null</td><td>5.0981</td><td>null</td><td>null</td><td>10</td><td>0</td></tr><tr><td>&quot;Female&quot;</td><td>&quot;C&quot;</td><td>&quot;A&quot;</td><td>4</td><td>7190.1</td><td>1450.0</td><td>null</td><td>3.557644</td><td>null</td><td>null</td><td>7</td><td>0</td></tr><tr><td>&quot;Female&quot;</td><td>&quot;B&quot;</td><td>&quot;C&quot;</td><td>4</td><td>1600.0</td><td>0.0</td><td>24000</td><td>4.0</td><td>35.5</td><td>943</td><td>2</td><td>0</td></tr><tr><td>&quot;Male&quot;</td><td>&quot;A&quot;</td><td>&quot;C&quot;</td><td>3</td><td>9893.0</td><td>1366.0</td><td>80000</td><td>5.0</td><td>null</td><td>null</td><td>10</td><td>0</td></tr><tr><td>&quot;Male&quot;</td><td>&quot;A&quot;</td><td>&quot;A&quot;</td><td>1</td><td>4230.0</td><td>0.0</td><td>69000</td><td>4.0</td><td>13.99</td><td>1885</td><td>10</td><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (68_899, 12)\n",
       "┌────────┬───────────────┬──────────────┬──────────────┬───┬──────────────┬──────┬──────┬──────────┐\n",
       "│ gender ┆ city_category ┆ employer_cat ┆ employer_cat ┆ … ┆ interest_rat ┆ emi  ┆ var1 ┆ approved │\n",
       "│ ---    ┆ ---           ┆ egory1       ┆ egory2       ┆   ┆ e            ┆ ---  ┆ ---  ┆ ---      │\n",
       "│ str    ┆ str           ┆ ---          ┆ ---          ┆   ┆ ---          ┆ i64  ┆ i64  ┆ i64      │\n",
       "│        ┆               ┆ str          ┆ i64          ┆   ┆ f64          ┆      ┆      ┆          │\n",
       "╞════════╪═══════════════╪══════════════╪══════════════╪═══╪══════════════╪══════╪══════╪══════════╡\n",
       "│ Female ┆ A             ┆ A            ┆ 4            ┆ … ┆ null         ┆ null ┆ 0    ┆ 0        │\n",
       "│ Male   ┆ A             ┆ C            ┆ 1            ┆ … ┆ 13.25        ┆ 953  ┆ 10   ┆ 0        │\n",
       "│ Male   ┆ C             ┆ C            ┆ 4            ┆ … ┆ null         ┆ null ┆ 0    ┆ 0        │\n",
       "│ Male   ┆ C             ┆ A            ┆ 4            ┆ … ┆ null         ┆ null ┆ 7    ┆ 0        │\n",
       "│ Male   ┆ A             ┆ A            ┆ 4            ┆ … ┆ null         ┆ null ┆ 10   ┆ 0        │\n",
       "│ …      ┆ …             ┆ …            ┆ …            ┆ … ┆ …            ┆ …    ┆ …    ┆ …        │\n",
       "│ Female ┆ A             ┆ A            ┆ 1            ┆ … ┆ null         ┆ null ┆ 10   ┆ 0        │\n",
       "│ Female ┆ C             ┆ A            ┆ 4            ┆ … ┆ null         ┆ null ┆ 7    ┆ 0        │\n",
       "│ Female ┆ B             ┆ C            ┆ 4            ┆ … ┆ 35.5         ┆ 943  ┆ 2    ┆ 0        │\n",
       "│ Male   ┆ A             ┆ C            ┆ 3            ┆ … ┆ null         ┆ null ┆ 10   ┆ 0        │\n",
       "│ Male   ┆ A             ┆ A            ┆ 1            ┆ … ┆ 13.99        ┆ 1885 ┆ 10   ┆ 0        │\n",
       "└────────┴───────────────┴──────────────┴──────────────┴───┴──────────────┴──────┴──────┴──────────┘"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import polars_ds.pipeline as pds_pipe\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class CustomPDSTransformer(BaseEstimator, TransformerMixin):\n",
    "\n",
    "    def __init__(self):\n",
    "        self.pipe = None\n",
    "\n",
    "    def fit(self, df, y=None):\n",
    "        # specify all the rules for the transform here\n",
    "        bp = (\n",
    "            pds_pipe.Blueprint(df, name = \"example\", target = \"approved\") \n",
    "            .lowercase() \n",
    "            .filter( \n",
    "                \"city_category is not null\" # or equivalently, you can do: pl.col(\"city_category\").is_not_null()\n",
    "            )\n",
    "            .select(cs.numeric() | cs.by_name([\"gender\", \"employer_category1\", \"city_category\", \"test_col\"]))\n",
    "            .linear_impute(features = [\"var1\", \"existing_emi\"], target = \"loan_period\") \n",
    "            .impute([\"existing_emi\"], method = \"median\")\n",
    "        )\n",
    "        self.pipe = bp.materialize()\n",
    "        return self\n",
    "\n",
    "    def transform(self, df, y=None):\n",
    "        return self.pipe.transform(df)\n",
    "\n",
    "# ---------------------------------------------------------------\n",
    "\n",
    "df = pl.read_parquet(\"../examples/dependency.parquet\")\n",
    "\n",
    "pipe = Pipeline(\n",
    "    steps=[\n",
    "        (\"CustomPDSTransformer\", CustomPDSTransformer())    \n",
    "    ]\n",
    ")\n",
    "df_transformed = pipe.fit_transform(df)\n",
    "df_transformed\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
